/*
 * Javlov - a Java toolkit for reinforcement learning with multi-agent support.
 * 
 * Copyright (c) 2009 Matthijs Snel
 * 
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package net.javlov;

/**
 * Learns the value function for a given policy, i.e. implements basic policy evaluation.
 * Policy improvement could take place by passing the agent a policy that depends on the
 * value function, instead of a fixed policy.
 * 
 * @author Matthijs Snel
 *
 */
public class TDAgent extends SimpleAgent {

	/**
	 * The value function.
	 */
	protected ValueFunction v;
	
	/**
	 * The value of the last state: V(s_t) or V(s).
	 */
	protected double lastValue;
	
	/**
	 * The value of the last TD error: {@code r + gamma*V(s') - V(s)}.
	 */
	private double lastTDerr;
	
	/**
	 * Discountfactor gamma.
	 */
	protected double gamma;
	
	/**
	 * Default constructor. Constructs TD(0) agent with tabular value function and gamma = 0.9.
	 */
	public TDAgent() {
		this( TabularValueFunction.getInstance(0), 0.9);
	}
	
	/**
	 * Constructs agent with given value function and gamma = 0.9.
	 * @param v the value function to use.
	 */
	public TDAgent(ValueFunction v) {
		this(v, 0.9);
	}
	
	/**
	 * Constructs agent with given value function and gamma.
	 * @param v the value function to use.
	 * @param gamma the discountfactor gamma e [0, 1].
	 */
	public TDAgent(ValueFunction v, double gamma) {
		setValueFunction(v);
		setGamma(gamma);
	}
	
	@Override
	public <T> Action doStep( State<T> s, double reward ) {
		updateValueFunction(s, reward);
		return super.doStep(s, reward);
	}

	@Override
	public <T> Action firstStep( State<T> s ) {
		lastValue = v.getValue(s);
		return super.firstStep(s);
	}
	
	@Override
	public void reset() {
		lastValue = 0;
		v.reset();
	}
	
	@Override
	public void init() {
		lastValue = 0;
		lastTDerr = 0;
		v.init();
	}
	
	public double getGamma() {
		return gamma;
	}

	/**
	 * Sets the discountfactor gamma, should be in the range [0, 1].
	 * @param gamma the discountfactor.
	 * @throws IllegalArgumentException when gamma is not in the range [0, 1].
	 */
	public void setGamma(double gamma) {
		if ( gamma < 0 || gamma > 1 )
			throw new IllegalArgumentException("Gamma needs to be between 0 and 1. Provided: " + gamma);
		this.gamma = gamma;
	}
	
	public double getLastTDError() {
		return lastTDerr;
	}
	
	public ValueFunction getValueFunction() {
		return v;
	}

	public void setValueFunction(ValueFunction v) {
		this.v = v;
	}
	
	protected void updateValueFunction(State s, double reward) {
		double targetValue = v.getValue(s);
		lastTDerr = reward + gamma*targetValue - lastValue;
		v.updatePrevious(lastTDerr);
		lastValue = targetValue;
	}
}
